Looks like there is a slightly higher rate of negative ratings for the high debt group, but also an even smaller increase in positive ratings.
data.frame(
"High_debt_version" = d.both_completed %>%
filter(high_debt_version == "true") %>%
pull(quality_post_task) %>%
revalue(c(
"-3"="Very Bad",
"-2"="Bad",
"-1"="Somewhat Bad",
"0"="Neutral",
"1"="Somewhat Good",
"2"="Good",
"3"="Very Good"
)),
"Low_debt_version" = d.both_completed %>%
filter(high_debt_version == "false") %>%
pull(quality_post_task) %>%
revalue(c(
"-3"="Very Bad",
"-2"="Bad",
"-1"="Somewhat Bad",
"0"="Neutral",
"1"="Somewhat Good",
"2"="Good",
"3"="Very Good"
))
) %>%
likert() %>%
plot(
type="density",
facet = TRUE,
)
As the data is collected from a likert scale we will use a cumulative family.
We include high_debt_verison as a predictor in our model as this variable represent the very effect we want to measure. We also include a varying intercept for each individual to prevent the model from learning too much from single participants with extreme measurements.
We iterate over the model until we have sane priors.
own_quality.with <- extendable_model(
base_name = "own_quality",
base_formula = "quality_post_task ~ 1 + (1 | session)",
base_priors = c(
prior(normal(0, 2.5), class = "Intercept"),
#prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd")
),
family = cumulative(),
data = d.both_completed,
base_control = list(adapt_delta = 0.95)
)
prior_summary(own_quality.with(only_priors= TRUE))
prior_summary(own_quality.with(sample_prior = "only"))
pp_check(own_quality.with(sample_prior = "only"), nsamples = 200, type = "bars")
sim.size <- 1000
sim.intercept <- rnorm(sim.size, 0, 2.5)
sim.beta <- rnorm(sim.size, 0, 1)
sim.beta.diff <- (plogis(sim.intercept + sim.beta) / plogis(sim.intercept) * 100) - 100
data.frame(x = sim.beta.diff) %>%
ggplot(aes(x)) +
geom_density() +
xlim(-150, 150) +
labs(
title = "Beta parameter prior influence",
x = "Estimate with beta as % of estimate without beta",
y = "Density"
)
We check the posterior distribution and can see that the model seems to have been able to fit the data well. Sampling seems to also have worked well as Rhat values are close to 1 and the sampling plots look nice.
pp_check(own_quality.with(), nsamples = 200, type = "bars")
summary(own_quality.with())
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + (1 | session)
## Data: as.data.frame(data) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.52 0.40 0.02 1.48 1.00 1322 1851
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept[1] -3.26 0.76 -4.87 -1.98 1.00 2854 2211
## Intercept[2] -0.85 0.36 -1.60 -0.16 1.00 3626 3194
## Intercept[3] 1.65 0.43 0.84 2.58 1.00 3782 2966
## Intercept[4] 2.37 0.52 1.44 3.47 1.00 4294 3480
## Intercept[5] 3.31 0.71 2.07 4.81 1.00 4956 3204
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
plot(own_quality.with(), ask = FALSE)
# default prior for monotonic predictor
edlvl_prior <- prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
# deafult beta prior
beta_prior <- prior(normal(0, 1), class = "b")
We use loo to check some possible extensions on the model.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
# New model(s)
own_quality.with("var_names_new_good.ratio", beta_prior),
own_quality.with("var_names_copied_good.ratio", beta_prior),
own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"), beta_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## own_quality.with() 0.0
## own_quality.with("var_names_copied_good.ratio", beta_prior) -0.2
## own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"), beta_prior) -0.3
## own_quality.with("var_names_new_good.ratio", beta_prior) -0.4
## se_diff
## own_quality.with() 0.0
## own_quality.with("var_names_copied_good.ratio", beta_prior) 0.7
## own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"), beta_prior) 0.7
## own_quality.with("var_names_new_good.ratio", beta_prior) 0.2
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("var_names_new_good.ratio", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.6 6.8
## p_loo 8.7 1.6
## looic 125.3 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 970
## (0.5, 0.7] (ok) 3 6.8% 434
## (0.7, 1] (bad) 1 2.3% 3394
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("var_names_copied_good.ratio", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.5 6.8
## p_loo 8.9 1.7
## looic 124.9 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 37 84.1% 1258
## (0.5, 0.7] (ok) 7 15.9% 454
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with(c("var_names_copied_good.ratio", "var_names_new_good.ratio"), beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.5 6.7
## p_loo 8.9 1.7
## looic 125.0 13.5
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 1135
## (0.5, 0.7] (ok) 6 13.6% 703
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
# New model(s)
own_quality.with("reused_logic_constructor", beta_prior),
own_quality.with("reused_logic_validation", beta_prior),
own_quality.with(c("reused_logic_validation", "reused_logic_constructor"), beta_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## own_quality.with("reused_logic_validation", beta_prior) 0.0
## own_quality.with(c("reused_logic_validation", "reused_logic_constructor"), beta_prior) -0.4
## own_quality.with("reused_logic_constructor", beta_prior) -1.5
## own_quality.with() -2.2
## se_diff
## own_quality.with("reused_logic_validation", beta_prior) 0.0
## own_quality.with(c("reused_logic_validation", "reused_logic_constructor"), beta_prior) 0.3
## own_quality.with("reused_logic_constructor", beta_prior) 1.1
## own_quality.with() 1.5
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("reused_logic_constructor", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.5 6.5
## p_loo 8.8 1.6
## looic 123.0 13.0
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 743
## (0.5, 0.7] (ok) 5 11.4% 2158
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("reused_logic_validation", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -60.0 6.1
## p_loo 8.4 1.5
## looic 120.1 12.2
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 1473
## (0.5, 0.7] (ok) 6 13.6% 744
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with(c("reused_logic_validation", "reused_logic_constructor"), beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -60.5 6.2
## p_loo 9.1 1.6
## looic 120.9 12.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 41 93.2% 859
## (0.5, 0.7] (ok) 3 6.8% 3502
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
# New model(s)
own_quality.with("equals.exists", beta_prior),
own_quality.with("hashcode.exists", beta_prior),
own_quality.with(c("hashcode.state", "equals.exists"), beta_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## own_quality.with() 0.0
## own_quality.with("equals.exists", beta_prior) -0.6
## own_quality.with("hashcode.exists", beta_prior) -0.7
## own_quality.with(c("hashcode.state", "equals.exists"), beta_prior) -0.7
## se_diff
## own_quality.with() 0.0
## own_quality.with("equals.exists", beta_prior) 0.2
## own_quality.with("hashcode.exists", beta_prior) 0.2
## own_quality.with(c("hashcode.state", "equals.exists"), beta_prior) 1.1
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("equals.exists", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.8 6.9
## p_loo 9.0 1.8
## looic 125.7 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 555
## (0.5, 0.7] (ok) 6 13.6% 513
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("hashcode.exists", beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.9 6.9
## p_loo 9.2 1.8
## looic 125.8 13.8
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 37 84.1% 1062
## (0.5, 0.7] (ok) 7 15.9% 499
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with(c("hashcode.state", "equals.exists"), beta_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.9 6.7
## p_loo 10.3 1.9
## looic 125.9 13.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 986
## (0.5, 0.7] (ok) 3 6.8% 521
## (0.7, 1] (bad) 1 2.3% 309
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
# New model(s)
own_quality.with("sonarqube_issues.s"),
own_quality.with("documentation")
)
loo_result[2]
## $diffs
## elpd_diff se_diff
## own_quality.with() 0.0 0.0
## own_quality.with("sonarqube_issues.s") -1.7 1.1
## own_quality.with("documentation") -2.7 1.4
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("sonarqube_issues.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -63.9 7.3
## p_loo 10.3 2.4
## looic 127.9 14.5
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 36 81.8% 1763
## (0.5, 0.7] (ok) 8 18.2% 238
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality.with("documentation")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.9 7.4
## p_loo 11.7 2.5
## looic 129.7 14.8
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 618
## (0.5, 0.7] (ok) 5 11.4% 315
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
We create a new base model based on what we learned from extending the previous model with different quality measurements. We take care to not include redundant quality indicators.
own_quality1.with <- extendable_model(
base_name = "own_quality1",
base_formula = "quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session)",
base_priors = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd")
),
family = cumulative(),
data = data.frame(d.both_completed),
base_control = list(adapt_delta = 0.95)
)
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
own_quality1.with(),
# New model(s)
own_quality1.with("work_domain"),
own_quality1.with("work_experience_programming.s"),
own_quality1.with("work_experience_java.s"),
own_quality1.with("education_field"),
own_quality1.with("mo(education_level)", edlvl_prior),
own_quality1.with("workplace_peer_review"),
own_quality1.with("workplace_td_tracking"),
own_quality1.with("workplace_pair_programming"),
own_quality1.with("workplace_coding_standards"),
own_quality1.with("scenario"),
own_quality1.with("group")
)
loo_result[2]
## $diffs
## elpd_diff se_diff
## own_quality1.with("mo(education_level)", edlvl_prior) 0.0 0.0
## own_quality.with() -1.1 3.0
## own_quality1.with("education_field") -2.8 2.7
## own_quality1.with("workplace_peer_review") -3.2 2.3
## own_quality1.with() -3.2 2.6
## own_quality1.with("workplace_coding_standards") -3.5 2.4
## own_quality1.with("group") -3.5 2.1
## own_quality1.with("workplace_pair_programming") -3.6 2.6
## own_quality1.with("scenario") -3.8 2.6
## own_quality1.with("workplace_td_tracking") -3.8 2.2
## own_quality1.with("work_experience_java.s") -4.0 2.2
## own_quality1.with("work_domain") -4.4 2.6
## own_quality1.with("work_experience_programming.s") -4.7 2.0
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.3 6.8
## p_loo 13.6 2.7
## looic 128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 661
## (0.5, 0.7] (ok) 2 4.5% 1140
## (0.7, 1] (bad) 2 4.5% 96
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("work_domain")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -65.5 7.2
## p_loo 17.0 3.2
## looic 131.0 14.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 504
## (0.5, 0.7] (ok) 6 13.6% 249
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("work_experience_programming.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -65.8 7.2
## p_loo 15.7 3.3
## looic 131.6 14.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 35 79.5% 467
## (0.5, 0.7] (ok) 8 18.2% 195
## (0.7, 1] (bad) 1 2.3% 137
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("work_experience_java.s")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -65.1 6.9
## p_loo 14.7 2.9
## looic 130.2 13.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 942
## (0.5, 0.7] (ok) 9 20.5% 326
## (0.7, 1] (bad) 1 2.3% 168
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -63.9 6.7
## p_loo 13.9 2.5
## looic 127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 892
## (0.5, 0.7] (ok) 3 6.8% 227
## (0.7, 1] (bad) 1 2.3% 133
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.1 7.2
## p_loo 12.9 2.6
## looic 122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 42 95.5% 770
## (0.5, 0.7] (ok) 2 4.5% 280
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.2 6.9
## p_loo 13.9 2.6
## looic 128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 935
## (0.5, 0.7] (ok) 5 11.4% 241
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_td_tracking")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.9 6.8
## p_loo 14.4 2.7
## looic 129.7 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 981
## (0.5, 0.7] (ok) 4 9.1% 350
## (0.7, 1] (bad) 2 4.5% 191
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_pair_programming")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.7 6.8
## p_loo 14.3 2.6
## looic 129.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 36 81.8% 938
## (0.5, 0.7] (ok) 8 18.2% 191
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_coding_standards")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.5 6.7
## p_loo 14.1 2.6
## looic 129.1 13.5
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 35 79.5% 590
## (0.5, 0.7] (ok) 8 18.2% 265
## (0.7, 1] (bad) 1 2.3% 173
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("scenario")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.8 6.7
## p_loo 14.2 2.6
## looic 129.7 13.3
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 582
## (0.5, 0.7] (ok) 5 11.4% 204
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("group")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.6 6.9
## p_loo 14.9 2.7
## looic 129.2 13.9
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 38 86.4% 538
## (0.5, 0.7] (ok) 4 9.1% 568
## (0.7, 1] (bad) 2 4.5% 258
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
own_quality1.with(),
own_quality1.with("mo(education_level)", edlvl_prior),
own_quality1.with("education_field"),
own_quality1.with("workplace_peer_review"),
# New model(s)
own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior),
own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior),
own_quality1.with(c("education_field", "workplace_peer_review"))
)
loo_result[2]
## $diffs
## elpd_diff
## own_quality1.with("mo(education_level)", edlvl_prior) 0.0
## own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior) -0.7
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior) -0.8
## own_quality.with() -1.1
## own_quality1.with("education_field") -2.8
## own_quality1.with("workplace_peer_review") -3.2
## own_quality1.with() -3.2
## own_quality1.with(c("education_field", "workplace_peer_review")) -3.4
## se_diff
## own_quality1.with("mo(education_level)", edlvl_prior) 0.0
## own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior) 0.4
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior) 0.3
## own_quality.with() 3.0
## own_quality1.with("education_field") 2.7
## own_quality1.with("workplace_peer_review") 2.3
## own_quality1.with() 2.6
## own_quality1.with(c("education_field", "workplace_peer_review")) 2.4
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.3 6.8
## p_loo 13.6 2.7
## looic 128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 661
## (0.5, 0.7] (ok) 2 4.5% 1140
## (0.7, 1] (bad) 2 4.5% 96
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.1 7.2
## p_loo 12.9 2.6
## looic 122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 42 95.5% 770
## (0.5, 0.7] (ok) 2 4.5% 280
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -63.9 6.7
## p_loo 13.9 2.5
## looic 127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 892
## (0.5, 0.7] (ok) 3 6.8% 227
## (0.7, 1] (bad) 1 2.3% 133
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.2 6.9
## p_loo 13.9 2.6
## looic 128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 935
## (0.5, 0.7] (ok) 5 11.4% 241
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.8 7.2
## p_loo 14.1 2.7
## looic 123.6 14.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 411
## (0.5, 0.7] (ok) 3 6.8% 925
## (0.7, 1] (bad) 2 4.5% 221
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.8 7.3
## p_loo 13.8 2.8
## looic 123.7 14.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 37 84.1% 1056
## (0.5, 0.7] (ok) 6 13.6% 394
## (0.7, 1] (bad) 1 2.3% 161
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("education_field", "workplace_peer_review"))`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.5 6.8
## p_loo 14.7 2.6
## looic 128.9 13.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 798
## (0.5, 0.7] (ok) 4 9.1% 257
## (0.7, 1] (bad) 1 2.3% 2782
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
loo_result <- loo(
# Benchmark model(s)
own_quality.with(),
own_quality1.with(),
own_quality1.with("mo(education_level)", edlvl_prior),
own_quality1.with("education_field"),
own_quality1.with("workplace_peer_review"),
own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior),
own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior),
# New model(s)
own_quality1.with(c("mo(education_level)", "education_field", "workplace_peer_review"), edlvl_prior)
)
loo_result[2]
## $diffs
## elpd_diff
## own_quality1.with("mo(education_level)", edlvl_prior) 0.0
## own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior) -0.7
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior) -0.8
## own_quality.with() -1.1
## own_quality1.with(c("mo(education_level)", "education_field", "workplace_peer_review"), edlvl_prior) -1.3
## own_quality1.with("education_field") -2.8
## own_quality1.with("workplace_peer_review") -3.2
## own_quality1.with() -3.2
## se_diff
## own_quality1.with("mo(education_level)", edlvl_prior) 0.0
## own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior) 0.4
## own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior) 0.3
## own_quality.with() 3.0
## own_quality1.with(c("mo(education_level)", "education_field", "workplace_peer_review"), edlvl_prior) 0.5
## own_quality1.with("education_field") 2.7
## own_quality1.with("workplace_peer_review") 2.3
## own_quality1.with() 2.6
loo_result[1]
## $loos
## $loos$`own_quality.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.2 6.8
## p_loo 8.2 1.6
## looic 124.4 13.6
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 34 77.3% 1043
## (0.5, 0.7] (ok) 10 22.7% 497
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with()`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.3 6.8
## p_loo 13.6 2.7
## looic 128.6 13.6
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 661
## (0.5, 0.7] (ok) 2 4.5% 1140
## (0.7, 1] (bad) 2 4.5% 96
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("mo(education_level)", edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.1 7.2
## p_loo 12.9 2.6
## looic 122.1 14.4
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 42 95.5% 770
## (0.5, 0.7] (ok) 2 4.5% 280
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("education_field")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -63.9 6.7
## p_loo 13.9 2.5
## looic 127.7 13.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 892
## (0.5, 0.7] (ok) 3 6.8% 227
## (0.7, 1] (bad) 1 2.3% 133
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with("workplace_peer_review")`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -64.2 6.9
## p_loo 13.9 2.6
## looic 128.5 13.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 935
## (0.5, 0.7] (ok) 5 11.4% 241
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("mo(education_level)", "education_field"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.8 7.2
## p_loo 14.1 2.7
## looic 123.6 14.4
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 39 88.6% 411
## (0.5, 0.7] (ok) 3 6.8% 925
## (0.7, 1] (bad) 2 4.5% 221
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("mo(education_level)", "workplace_peer_review"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -61.8 7.3
## p_loo 13.8 2.8
## looic 123.7 14.7
## ------
## Monte Carlo SE of elpd_loo is NA.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 37 84.1% 1056
## (0.5, 0.7] (ok) 6 13.6% 394
## (0.7, 1] (bad) 1 2.3% 161
## (1, Inf) (very bad) 0 0.0% <NA>
## See help('pareto-k-diagnostic') for details.
##
## $loos$`own_quality1.with(c("mo(education_level)", "education_field", "workplace_peer_review"), edlvl_prior)`
##
## Computed from 4000 by 44 log-likelihood matrix
##
## Estimate SE
## elpd_loo -62.4 7.3
## p_loo 14.6 2.8
## looic 124.8 14.7
## ------
## Monte Carlo SE of elpd_loo is 0.1.
##
## Pareto k diagnostic values:
## Count Pct. Min. n_eff
## (-Inf, 0.5] (good) 40 90.9% 1106
## (0.5, 0.7] (ok) 4 9.1% 175
## (0.7, 1] (bad) 0 0.0% <NA>
## (1, Inf) (very bad) 0 0.0% <NA>
##
## All Pareto k estimates are ok (k < 0.7).
## See help('pareto-k-diagnostic') for details.
We pick some of our top performing models as candidates and inspect them closer.
The candidate models are named and listed in order of complexity.
We select the simplest model as a baseline.
own_quality0 <- brm(
"quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session)",
prior = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd")
),
family = cumulative(),
data = as.data.frame(d.both_completed),
file = "fits/own_quality0",
file_refit = "on_change",
control = list(adapt_delta = 0.95),
seed = 20210421
)
summary(own_quality0)
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.55 0.42 0.02 1.57 1.00 1597 2169
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1] -4.16 1.21 -6.61 -1.84 1.00 4785
## Intercept[2] -1.64 1.04 -3.69 0.43 1.00 5536
## Intercept[3] 1.08 1.04 -0.91 3.23 1.00 5148
## Intercept[4] 1.86 1.06 -0.15 4.06 1.00 5386
## Intercept[5] 2.84 1.15 0.66 5.25 1.00 5107
## var_names_copied_good.ratio 0.18 0.69 -1.16 1.51 1.00 5238
## var_names_new_good.ratio -0.19 0.76 -1.66 1.31 1.00 5848
## reused_logic_validationfalse -1.29 0.69 -2.65 0.07 1.00 5620
## equals.existsFALSE -0.12 0.60 -1.30 1.05 1.00 5595
## sonarqube_issues.s 0.14 0.33 -0.55 0.77 1.00 5153
## documentationIncorrect -0.08 0.70 -1.45 1.33 1.00 4948
## documentationNone -0.32 0.67 -1.61 1.00 1.00 5163
## Tail_ESS
## Intercept[1] 2890
## Intercept[2] 3421
## Intercept[3] 3291
## Intercept[4] 2967
## Intercept[5] 3177
## var_names_copied_good.ratio 3458
## var_names_new_good.ratio 3368
## reused_logic_validationfalse 3143
## equals.existsFALSE 2682
## sonarqube_issues.s 3093
## documentationIncorrect 3273
## documentationNone 3166
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(own_quality0)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 -0.003597013 0.5691377 -1.2575950 1.2396757
## 6033d90a5af2c702367b3a96 0.014789321 0.5393521 -1.1573882 1.2094220
## 6034fc165af2c702367b3a98 0.217146642 0.6379817 -0.9111772 1.8831193
## 603500725af2c702367b3a99 0.217880628 0.6287807 -0.9036990 1.7542683
## 603f97625af2c702367b3a9d 0.195295978 0.6159302 -0.9047822 1.7731190
## 603fd5d95af2c702367b3a9e 0.211933782 0.6094602 -0.8348789 1.8124290
## 60409b7b5af2c702367b3a9f -0.291503792 0.6534897 -1.9900278 0.7329610
## 604b82b5a7718fbed181b336 -0.118956619 0.5970882 -1.5933948 1.0402783
## 6050c1bf856f36729d2e5218 -0.381506746 0.7043987 -2.1840885 0.5912000
## 6050e1e7856f36729d2e5219 0.300178134 0.6807822 -0.7088807 2.1312623
## 6055fdc6856f36729d2e521b -0.095024017 0.5870477 -1.4352780 1.0614585
## 60589862856f36729d2e521f 0.017417753 0.6351805 -1.3912627 1.4370708
## 605afa3a856f36729d2e5222 0.111599771 0.5979707 -1.0760257 1.5341443
## 605c8bc6856f36729d2e5223 0.115222936 0.5451251 -0.9293503 1.4080110
## 605f3f2d856f36729d2e5224 -0.294684438 0.6694775 -2.0817340 0.6980392
## 605f46c3856f36729d2e5225 -0.454960073 0.7527216 -2.4583685 0.4801064
## 60605337856f36729d2e5226 -0.042958797 0.5984055 -1.3864395 1.1934620
## 60609ae6856f36729d2e5228 -0.107330078 0.5570589 -1.4428330 0.9695812
## 6061ce91856f36729d2e522e 0.265634212 0.6361855 -0.7333939 1.8960458
## 6061f106856f36729d2e5231 0.024195154 0.5225656 -1.1240073 1.2148825
## 6068ea9f856f36729d2e523e 0.225063116 0.6117656 -0.7948535 1.8227282
## 6075ab05856f36729d2e5247 -0.459722059 0.7565080 -2.4875737 0.4958027
plot(own_quality0, ask = FALSE)
pp_check(own_quality0, nsamples = 200, type = "bars")
We select the best performing model with one variable.
own_quality1 <- brm(
"quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session) +
mo(education_level)",
prior = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd"),
prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
),
family = cumulative(),
data = as.data.frame(d.both_completed),
file = "fits/own_quality1",
file_refit = "on_change",
control = list(adapt_delta = 0.95),
seed = 20210421
)
summary(own_quality1)
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level)
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.41 0.34 0.01 1.27 1.00 2197 2087
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1] -5.53 1.34 -8.30 -3.06 1.00 5225
## Intercept[2] -2.99 1.17 -5.29 -0.76 1.00 5526
## Intercept[3] 0.05 1.12 -2.10 2.23 1.00 6353
## Intercept[4] 0.90 1.14 -1.29 3.13 1.00 6385
## Intercept[5] 1.94 1.23 -0.41 4.44 1.00 6717
## var_names_copied_good.ratio 0.31 0.69 -1.02 1.66 1.00 6468
## var_names_new_good.ratio -0.21 0.74 -1.61 1.23 1.00 6675
## reused_logic_validationfalse -1.14 0.68 -2.45 0.22 1.00 7016
## equals.existsFALSE 0.01 0.59 -1.14 1.17 1.00 6521
## sonarqube_issues.s 0.33 0.33 -0.35 0.94 1.00 6424
## documentationIncorrect -0.22 0.72 -1.63 1.18 1.00 6196
## documentationNone -0.36 0.68 -1.71 0.99 1.00 5939
## moeducation_level -0.62 0.25 -1.10 -0.14 1.00 4139
## Tail_ESS
## Intercept[1] 3039
## Intercept[2] 3164
## Intercept[3] 3301
## Intercept[4] 3551
## Intercept[5] 3625
## var_names_copied_good.ratio 3244
## var_names_new_good.ratio 3271
## reused_logic_validationfalse 3386
## equals.existsFALSE 3076
## sonarqube_issues.s 3303
## documentationIncorrect 2935
## documentationNone 2762
## moeducation_level 2796
##
## Simplex Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1] 0.22 0.12 0.03 0.49 1.00 8404
## moeducation_level1[2] 0.29 0.15 0.05 0.61 1.00 7341
## moeducation_level1[3] 0.24 0.14 0.04 0.55 1.00 6974
## moeducation_level1[4] 0.26 0.14 0.04 0.56 1.00 6785
## Tail_ESS
## moeducation_level1[1] 2696
## moeducation_level1[2] 2904
## moeducation_level1[3] 2608
## moeducation_level1[4] 2647
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(own_quality1)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.06345376 0.4485564 -0.8276452 1.1370718
## 6033d90a5af2c702367b3a96 -0.05426508 0.4674512 -1.1435667 0.8955601
## 6034fc165af2c702367b3a98 0.18612804 0.5230054 -0.6624199 1.5735883
## 603500725af2c702367b3a99 0.06066618 0.4929435 -0.9108607 1.2559880
## 603f97625af2c702367b3a9d 0.15770562 0.5074407 -0.6919706 1.4723548
## 603fd5d95af2c702367b3a9e 0.08487268 0.4619562 -0.7802987 1.2096927
## 60409b7b5af2c702367b3a9f -0.16826729 0.5084530 -1.4643742 0.6388627
## 604b82b5a7718fbed181b336 0.02327163 0.4754285 -0.9840679 1.1390013
## 6050c1bf856f36729d2e5218 -0.19337378 0.5084320 -1.5603862 0.5841307
## 6050e1e7856f36729d2e5219 0.14032548 0.5088986 -0.7286702 1.4890807
## 6055fdc6856f36729d2e521b -0.07091407 0.4677973 -1.2335113 0.8417580
## 60589862856f36729d2e521f 0.03895203 0.5036893 -1.0144965 1.2125065
## 605afa3a856f36729d2e5222 0.15054573 0.5138011 -0.7006577 1.4565383
## 605c8bc6856f36729d2e5223 0.20344464 0.5116691 -0.5863046 1.5952393
## 605f3f2d856f36729d2e5224 -0.13638148 0.4970087 -1.3862195 0.7279973
## 605f46c3856f36729d2e5225 -0.15981704 0.5134788 -1.5232868 0.7281112
## 60605337856f36729d2e5226 0.02742256 0.4771778 -1.0105487 1.0763487
## 60609ae6856f36729d2e5228 -0.07947723 0.4624653 -1.2304770 0.8277921
## 6061ce91856f36729d2e522e 0.12274911 0.4946236 -0.7876618 1.3457835
## 6061f106856f36729d2e5231 -0.06599802 0.4681963 -1.1701008 0.8540004
## 6068ea9f856f36729d2e523e 0.03282397 0.4496302 -0.8559524 1.0725408
## 6075ab05856f36729d2e5247 -0.18843313 0.5089148 -1.5473897 0.6357123
plot(own_quality1, ask = FALSE)
pp_check(own_quality1, nsamples = 200, type = "bars")
We select the best performing model with two variables.
own_quality2 <- brm(
"quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session) +
mo(education_level) +
education_field",
prior = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd"),
prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
),
family = cumulative(),
data = as.data.frame(d.both_completed),
file = "fits/own_quality2",
file_refit = "on_change",
control = list(adapt_delta = 0.95),
seed = 20210421
)
summary(own_quality2)
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level) + education_field
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.43 0.35 0.01 1.30 1.00 2009 2255
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept[1] -5.39 1.40 -8.17 -2.68 1.00
## Intercept[2] -2.81 1.25 -5.26 -0.33 1.00
## Intercept[3] 0.28 1.22 -2.04 2.65 1.00
## Intercept[4] 1.14 1.23 -1.19 3.54 1.00
## Intercept[5] 2.17 1.31 -0.31 4.76 1.00
## var_names_copied_good.ratio 0.29 0.69 -1.05 1.67 1.00
## var_names_new_good.ratio -0.14 0.77 -1.64 1.34 1.00
## reused_logic_validationfalse -1.15 0.67 -2.42 0.14 1.00
## equals.existsFALSE 0.03 0.58 -1.08 1.20 1.00
## sonarqube_issues.s 0.33 0.33 -0.33 0.96 1.00
## documentationIncorrect -0.24 0.71 -1.65 1.15 1.00
## documentationNone -0.33 0.66 -1.64 0.96 1.00
## education_fieldInteractionDesign -0.56 0.84 -2.19 1.11 1.00
## education_fieldNone 0.06 0.85 -1.58 1.75 1.00
## education_fieldSoftwareEngineering 0.18 0.59 -0.99 1.31 1.00
## moeducation_level -0.60 0.25 -1.11 -0.12 1.00
## Bulk_ESS Tail_ESS
## Intercept[1] 4188 2961
## Intercept[2] 4680 3129
## Intercept[3] 5223 3231
## Intercept[4] 5385 3378
## Intercept[5] 5708 3296
## var_names_copied_good.ratio 5565 3269
## var_names_new_good.ratio 5338 3162
## reused_logic_validationfalse 6004 3163
## equals.existsFALSE 6160 2950
## sonarqube_issues.s 5660 3112
## documentationIncorrect 5071 2902
## documentationNone 5404 3087
## education_fieldInteractionDesign 7348 3272
## education_fieldNone 6628 2752
## education_fieldSoftwareEngineering 6281 3545
## moeducation_level 3832 3028
##
## Simplex Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1] 0.22 0.12 0.04 0.51 1.00 5585
## moeducation_level1[2] 0.28 0.15 0.04 0.61 1.00 6360
## moeducation_level1[3] 0.24 0.13 0.04 0.54 1.00 6496
## moeducation_level1[4] 0.26 0.14 0.04 0.56 1.00 5996
## Tail_ESS
## moeducation_level1[1] 2578
## moeducation_level1[2] 2651
## moeducation_level1[3] 2918
## moeducation_level1[4] 2952
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(own_quality2)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.05862681 0.4857455 -0.9121941 1.1835278
## 6033d90a5af2c702367b3a96 -0.07271323 0.4741916 -1.2154645 0.8372287
## 6034fc165af2c702367b3a98 0.19688443 0.5438811 -0.6333864 1.6680245
## 603500725af2c702367b3a99 0.06398798 0.4689289 -0.8765425 1.1960245
## 603f97625af2c702367b3a9d 0.16181150 0.5254654 -0.7154987 1.5184058
## 603fd5d95af2c702367b3a9e 0.07312910 0.4579353 -0.8192655 1.1168413
## 60409b7b5af2c702367b3a9f -0.17697272 0.5095151 -1.4989220 0.6204594
## 604b82b5a7718fbed181b336 -0.01560080 0.4669579 -1.0608800 1.0319015
## 6050c1bf856f36729d2e5218 -0.20436839 0.5376292 -1.6021532 0.6587857
## 6050e1e7856f36729d2e5219 0.16060710 0.5382503 -0.7046126 1.6168415
## 6055fdc6856f36729d2e521b -0.08492602 0.4881045 -1.2843642 0.8441007
## 60589862856f36729d2e521f 0.04474436 0.5209253 -0.9480323 1.2590313
## 605afa3a856f36729d2e5222 0.16971821 0.5315386 -0.7237819 1.5431340
## 605c8bc6856f36729d2e5223 0.20534044 0.5243361 -0.5873331 1.6182595
## 605f3f2d856f36729d2e5224 -0.14280855 0.5054777 -1.4510252 0.7291005
## 605f46c3856f36729d2e5225 -0.18680275 0.5255168 -1.5663795 0.6696721
## 60605337856f36729d2e5226 0.01504140 0.4987200 -1.0549720 1.1020085
## 60609ae6856f36729d2e5228 -0.07851222 0.4903465 -1.2815460 0.8457675
## 6061ce91856f36729d2e522e 0.12313348 0.4963280 -0.7650773 1.3863915
## 6061f106856f36729d2e5231 -0.06291447 0.4714056 -1.1808842 0.8799728
## 6068ea9f856f36729d2e523e 0.03559989 0.5030245 -1.0206005 1.2449750
## 6075ab05856f36729d2e5247 -0.12962967 0.5227119 -1.4453770 0.8034020
plot(own_quality2, ask = FALSE)
pp_check(own_quality2, nsamples = 200, type = "bars")
We select the best performing model with three variables.
own_quality3 <- brm(
"quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session) +
mo(education_level) +
education_field +
workplace_peer_review",
prior = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd"),
prior(dirichlet(2), class = "simo", coef = "moeducation_level1")
),
family = cumulative(),
data = as.data.frame(d.both_completed),
file = "fits/own_quality3",
file_refit = "on_change",
control = list(adapt_delta = 0.95),
seed = 20210421
)
summary(own_quality3)
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session) + mo(education_level) + education_field + workplace_peer_review
## Data: as.data.frame(d.both_completed) (Number of observations: 44)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 22)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.44 0.36 0.02 1.32 1.00 1874 2570
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat
## Intercept[1] -5.42 1.45 -8.29 -2.67 1.00
## Intercept[2] -2.86 1.31 -5.54 -0.28 1.00
## Intercept[3] 0.24 1.23 -2.22 2.67 1.00
## Intercept[4] 1.10 1.26 -1.41 3.57 1.00
## Intercept[5] 2.16 1.30 -0.48 4.78 1.00
## var_names_copied_good.ratio 0.31 0.68 -1.01 1.63 1.00
## var_names_new_good.ratio -0.15 0.77 -1.67 1.36 1.00
## reused_logic_validationfalse -1.15 0.71 -2.55 0.25 1.00
## equals.existsFALSE 0.03 0.60 -1.16 1.21 1.00
## sonarqube_issues.s 0.35 0.34 -0.33 0.98 1.00
## documentationIncorrect -0.24 0.73 -1.67 1.19 1.00
## documentationNone -0.32 0.67 -1.64 0.99 1.00
## education_fieldInteractionDesign -0.54 0.84 -2.19 1.11 1.00
## education_fieldNone 0.06 0.86 -1.64 1.71 1.00
## education_fieldSoftwareEngineering 0.19 0.59 -0.97 1.34 1.00
## workplace_peer_reviewfalse -0.09 0.59 -1.26 1.08 1.00
## moeducation_level -0.60 0.26 -1.13 -0.11 1.00
## Bulk_ESS Tail_ESS
## Intercept[1] 5297 2818
## Intercept[2] 5009 3112
## Intercept[3] 4842 3325
## Intercept[4] 5237 3193
## Intercept[5] 5204 3135
## var_names_copied_good.ratio 6424 3457
## var_names_new_good.ratio 6238 3236
## reused_logic_validationfalse 6542 3272
## equals.existsFALSE 5886 2923
## sonarqube_issues.s 5454 2842
## documentationIncorrect 5077 2676
## documentationNone 6277 3190
## education_fieldInteractionDesign 6726 3013
## education_fieldNone 7045 3195
## education_fieldSoftwareEngineering 6157 3265
## workplace_peer_reviewfalse 5420 2880
## moeducation_level 4061 3120
##
## Simplex Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## moeducation_level1[1] 0.22 0.13 0.03 0.50 1.00 6699
## moeducation_level1[2] 0.28 0.14 0.05 0.59 1.00 7150
## moeducation_level1[3] 0.24 0.14 0.04 0.54 1.00 6450
## moeducation_level1[4] 0.26 0.14 0.05 0.55 1.00 7011
## Tail_ESS
## moeducation_level1[1] 2431
## moeducation_level1[2] 2803
## moeducation_level1[3] 2737
## moeducation_level1[4] 3535
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(own_quality3)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033d69a5af2c702367b3a95 0.07491700 0.4911729 -0.9430459 1.2504775
## 6033d90a5af2c702367b3a96 -0.06422939 0.4855022 -1.2000190 0.8806904
## 6034fc165af2c702367b3a98 0.18351701 0.5363674 -0.6744247 1.6376418
## 603500725af2c702367b3a99 0.07203840 0.4972385 -0.9593952 1.2888728
## 603f97625af2c702367b3a9d 0.16653041 0.5304650 -0.7274883 1.5075223
## 603fd5d95af2c702367b3a9e 0.07430359 0.4803014 -0.8234218 1.1933480
## 60409b7b5af2c702367b3a9f -0.19549489 0.5278701 -1.5943530 0.6346203
## 604b82b5a7718fbed181b336 -0.02485149 0.4916338 -1.1153138 1.0381780
## 6050c1bf856f36729d2e5218 -0.22129918 0.5557692 -1.6922722 0.6191403
## 6050e1e7856f36729d2e5219 0.14790831 0.5310898 -0.7941579 1.5089705
## 6055fdc6856f36729d2e521b -0.09351285 0.5250341 -1.4110362 0.9013951
## 60589862856f36729d2e521f 0.03825752 0.5241489 -1.0470568 1.2377180
## 605afa3a856f36729d2e5222 0.16073087 0.5085764 -0.6865203 1.5469607
## 605c8bc6856f36729d2e5223 0.20357679 0.5468986 -0.6734016 1.6331773
## 605f3f2d856f36729d2e5224 -0.15795929 0.5378507 -1.5687513 0.7968925
## 605f46c3856f36729d2e5225 -0.20337705 0.5293703 -1.6190217 0.6239888
## 60605337856f36729d2e5226 0.02294887 0.4898817 -1.0002530 1.1326100
## 60609ae6856f36729d2e5228 -0.06817839 0.4936718 -1.2820645 0.9673343
## 6061ce91856f36729d2e522e 0.12301079 0.5143776 -0.8032908 1.4377078
## 6061f106856f36729d2e5231 -0.06028052 0.4667851 -1.1729798 0.8388732
## 6068ea9f856f36729d2e523e 0.03857722 0.5145712 -0.9961772 1.2367408
## 6075ab05856f36729d2e5247 -0.14589928 0.5416454 -1.5395385 0.8075027
plot(own_quality3, ask = FALSE)
pp_check(own_quality3, nsamples = 200, type = "bars")
All candidate models look nice, none is significantly better than the others, we will proceed the model containing work experince as it otherwise ourd be added in the next step: own_quality0
Some participants did only complete one scenario. Those has been excluded from the initial dataset to improve sampling of the models. We do however want to use all data we can and will therefore try to fit the model with the complete dataset.
own_quality0.all <- brm(
"quality_post_task ~ 1 +
var_names_copied_good.ratio +
var_names_new_good.ratio +
reused_logic_validation +
equals.exists +
sonarqube_issues.s +
documentation +
(1 | session)",
prior = c(
prior(normal(0, 2.5), class = "Intercept"),
prior(normal(0, 1), class = "b"),
prior(exponential(1), class = "sd")
),
family = cumulative(),
data = as.data.frame(d.completed),
file = "fits/own_quality0.all",
file_refit = "on_change",
control = list(adapt_delta = 0.95),
seed = 20210421
)
summary(own_quality0.all)
## Family: cumulative
## Links: mu = logit; disc = identity
## Formula: quality_post_task ~ 1 + var_names_copied_good.ratio + var_names_new_good.ratio + reused_logic_validation + equals.exists + sonarqube_issues.s + documentation + (1 | session)
## Data: as.data.frame(d.completed) (Number of observations: 51)
## Samples: 4 chains, each with iter = 2000; warmup = 1000; thin = 1;
## total post-warmup samples = 4000
##
## Group-Level Effects:
## ~session (Number of levels: 29)
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept) 0.50 0.39 0.01 1.43 1.00 1284 1779
##
## Population-Level Effects:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept[1] -4.81 1.25 -7.39 -2.48 1.00 3799
## Intercept[2] -3.53 1.07 -5.71 -1.45 1.00 4382
## Intercept[3] -1.46 0.98 -3.43 0.45 1.00 4286
## Intercept[4] 1.43 0.99 -0.48 3.47 1.00 4279
## Intercept[5] 2.17 1.02 0.20 4.24 1.00 4288
## Intercept[6] 3.12 1.11 0.99 5.41 1.00 4863
## var_names_copied_good.ratio 0.19 0.66 -1.10 1.52 1.00 4591
## var_names_new_good.ratio -0.04 0.77 -1.53 1.46 1.00 5362
## reused_logic_validationfalse -1.31 0.66 -2.62 -0.03 1.00 4370
## equals.existsFALSE -0.17 0.56 -1.25 0.97 1.00 4669
## sonarqube_issues.s 0.09 0.31 -0.54 0.68 1.00 4245
## documentationIncorrect -0.19 0.67 -1.51 1.12 1.00 4038
## documentationNone -0.33 0.61 -1.53 0.91 1.00 4519
## Tail_ESS
## Intercept[1] 2853
## Intercept[2] 3324
## Intercept[3] 3159
## Intercept[4] 3267
## Intercept[5] 3268
## Intercept[6] 3112
## var_names_copied_good.ratio 3473
## var_names_new_good.ratio 3374
## reused_logic_validationfalse 3297
## equals.existsFALSE 2864
## sonarqube_issues.s 3069
## documentationIncorrect 3285
## documentationNone 3238
##
## Family Specific Parameters:
## Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## disc 1.00 0.00 1.00 1.00 1.00 4000 4000
##
## Samples were drawn using sample(hmc). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
ranef(own_quality0.all)
## $session
## , , Intercept
##
## Estimate Est.Error Q2.5 Q97.5
## 6033c6fc5af2c702367b3a93 -0.349289242 0.7795327 -2.5428940 0.6959831
## 6033d69a5af2c702367b3a95 0.026530756 0.5260593 -1.0998937 1.2147825
## 6033d90a5af2c702367b3a96 0.029703767 0.5164378 -1.0819540 1.2027353
## 6034fc165af2c702367b3a98 0.238125196 0.6090597 -0.7396881 1.8299838
## 603500725af2c702367b3a99 0.231422198 0.5973404 -0.7560744 1.7214345
## 603f84f15af2c702367b3a9b 0.005333960 0.5695396 -1.2383455 1.2915118
## 603f97625af2c702367b3a9d 0.210284903 0.5875872 -0.7253746 1.7977908
## 603fd5d95af2c702367b3a9e 0.223026533 0.5820774 -0.7166333 1.7001420
## 60409b7b5af2c702367b3a9f -0.193239496 0.5464399 -1.5337175 0.7521659
## 604b82b5a7718fbed181b336 -0.067691678 0.5589897 -1.3786600 1.0439938
## 604f1239a7718fbed181b33f 0.015989792 0.5593005 -1.2059358 1.2437528
## 6050c1bf856f36729d2e5218 -0.273141168 0.5758688 -1.7497000 0.6201582
## 6050e1e7856f36729d2e5219 0.278264995 0.6353341 -0.6588195 1.9282408
## 6055fdc6856f36729d2e521b -0.053030916 0.5364673 -1.3167242 1.0154745
## 60579f2a856f36729d2e521e -0.218149016 0.6219092 -1.8630270 0.7897970
## 60589862856f36729d2e521f 0.039798725 0.5767338 -1.1643550 1.3791355
## 605a30a7856f36729d2e5221 0.001669608 0.5450172 -1.1957662 1.1962413
## 605afa3a856f36729d2e5222 0.122957724 0.5181212 -0.8428584 1.3465340
## 605c8bc6856f36729d2e5223 0.130266187 0.5412859 -0.9127264 1.4678885
## 605f3f2d856f36729d2e5224 -0.200733573 0.5628212 -1.6064560 0.7337097
## 605f46c3856f36729d2e5225 -0.343036209 0.6548436 -2.1283313 0.5793587
## 60605337856f36729d2e5226 -0.004899536 0.5175498 -1.1205360 1.1506197
## 60609ae6856f36729d2e5228 -0.070159489 0.5107317 -1.2259107 1.0033435
## 6061ce91856f36729d2e522e 0.242088562 0.5935935 -0.6770190 1.8501147
## 6061f106856f36729d2e5231 0.071695997 0.5207193 -0.9846461 1.2769810
## 60672faa856f36729d2e523c 0.037295031 0.5385528 -1.0737692 1.2501955
## 6068ea9f856f36729d2e523e 0.215376027 0.5700790 -0.6903914 1.6900623
## 606db69d856f36729d2e5243 0.005385041 0.5639127 -1.2658303 1.2829358
## 6075ab05856f36729d2e5247 -0.365414846 0.6426471 -2.0586880 0.4860709
plot(own_quality0.all, ask = FALSE)
pp_check(own_quality0.all, nsamples = 200, type = "bars")
This means that our final model, with all data points and experience predictors, is own_quality0.all
To begin interpreting the model we look at how it’s parameters were estimated. As our research is focused on how the outcome of the model is effected we will mainly analyze the \(\beta\) parameters.
mcmc_areas(own_quality0.all,
pars = c(
"b_var_names_copied_good.ratio",
"b_var_names_new_good.ratio",
"b_reused_logic_validationfalse",
"b_equals.existsFALSE",
"b_sonarqube_issues.s",
"b_documentationIncorrect",
"b_documentationNone"
),
prob = 0.95) + scale_y_discrete() +
scale_y_discrete(labels=c(
"Ratio of good copied var names",
"Ratio of good new var names",
"Duplicated validation logic",
"Missing equals implementation",
"Amount of sonarqube issues",
"Incorrect documentation",
"No documentation"
)) +
ggtitle("Beta parameters densities in self assesed quality model", subtitle = "Shaded region marks 95% of the density. Line marks the median")
As we have a low of effects playing small roles we will simulate two scenarios, one where the developer, according to us does well and one where the developers doesnt do so well and se if the participant rating approves with us.
scale_programming_experience <- function(x) {
(x - mean(d.completed$work_experience_programming))/ sd(d.completed$work_experience_programming)
}
unscale_programming_experience <- function(x) {
x * sd(d.completed$work_experience_programming) + mean(d.completed$work_experience_programming)
}
post_settings <- data_frame(
var_names_copied_good.ratio = c(0.9, 0.5),
var_names_new_good.ratio = c(0.9, 0.5),
reused_logic_validation = c("true", "false"),
equals.exists = c("TRUE", "FALSE"),
sonarqube_issues.s = c(-1, 1),
documentation = c("Correct", "Incorrect"),
session = NA
)
post <- posterior_predict(own_quality0.all, newdata = post_settings) %>%
melt(value.name = "estimate", varnames = c("sample_number", "settings_id")) %>%
left_join(
rowid_to_column(post_settings, var= "settings_id"),
by = "settings_id"
) %>%
mutate(submission = revalue(reused_logic_validation, c("true" = "Good", "false" = "Bad"))) %>%
select(
estimate,
submission
)
post.nice <- post %>% mutate_at("estimate", function(x) revalue(as.ordered(x), c(
"1"="Very Bad",
"2"="Bad",
"3"="Somewhat Bad",
"4"="Neutral",
"5"="Somewhat Good",
"6"="Good",
"7"="Very Good"
)))
data.frame(
Bad_submission = post.nice %>%
filter(submission == "Bad") %>%
pull(estimate),
Good_submission = post.nice %>%
filter(submission == "Good") %>%
pull(estimate)
) %>%
likert() %>%
plot(
type="density",
facet = TRUE,
)